% Variable To Be Explored: Investigate increasing MaxEpochs from 5 to 10 to 20
% Variable To Be Explored: Use small, UN-augmented data set
% All other variables held constant
% Use established "layers" modified pretrained alexnet network
epoch_model_small_data_accuracies = [];
epoch_models = ["5-epoch", "10-epoch", "20-epoch"];
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_small, ...
'Plots','training-progress');
five_epoch_accuracy = model_optimization_explorer (trainingSet_small, validationSet_small, testSet_small, model_options, layers);
The validation accuracy is: 100.00 %
epoch_model_small_data_accuracies = [epoch_model_small_data_accuracies, five_epoch_accuracy]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_small, ...
'Plots','training-progress');
ten_epoch_accuracy = model_optimization_explorer (trainingSet_small, validationSet_small, testSet_small, model_options, layers);
The validation accuracy is: 100.00 %
epoch_model_small_data_accuracies = [epoch_model_small_data_accuracies, ten_epoch_accuracy]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_small, ...
'Plots','training-progress');
twenty_epoch_accuracy = model_optimization_explorer (trainingSet_small, validationSet_small, testSet_small, model_options, layers);
The validation accuracy is: 83.33 %
epoch_model_small_data_accuracies = [epoch_model_small_data_accuracies twenty_epoch_accuracy]; % Capture accuracy value
disp(epoch_models)
"5-epoch" "10-epoch" "20-epoch"
disp(epoch_model_small_data_accuracies)
% Variable To Be Explored: Investigate increasing MaxEpochs from 5 to 10 to 20
% Variable To Be Explored: Use large, UN-augmented data set
% All other variables held constant
% Use established "layers" modified pretrained alexnet network
epoch_model_large_data_accuracies = [];
epoch_models = ["5-epoch", "10-epoch", "20-epoch"];
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large, ...
'Plots','training-progress');
five_epoch_accuracy = model_optimization_explorer (trainingSet_large, validationSet_large, testSet_large, model_options, layers);
The validation accuracy is: 94.00 %
epoch_model_large_data_accuracies = [epoch_model_large_data_accuracies, five_epoch_accuracy]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large, ...
'Plots','training-progress');
ten_epoch_accuracy = model_optimization_explorer (trainingSet_large, validationSet_large, testSet_large, model_options, layers);
The validation accuracy is: 92.67 %
epoch_model_large_data_accuracies = [epoch_model_large_data_accuracies, ten_epoch_accuracy]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large, ...
'Plots','training-progress');
twenty_epoch_accuracy = model_optimization_explorer (trainingSet_large, validationSet_large, testSet_large, model_options, layers);
The validation accuracy is: 94.00 %
epoch_model_large_data_accuracies = [epoch_model_large_data_accuracies twenty_epoch_accuracy]; % Capture accuracy value
disp(epoch_models)
"5-epoch" "10-epoch" "20-epoch"
disp(epoch_model_large_data_accuracies)
% Variable To Be Explored: Various (%) values for partitioning the dataset into training, validation, and test sets
% Variable Selected/Locked After Testing: MaxEpochs held at 10
% Variable Selected/Locked After Testing: Use large, UN-augmented data set
% All other variables held constant
% Use established "layers" modified pretrained alexnet network
split_models_accuracies = [];
split_models = ["60/20/20-split", "70/15/15-split", "80/10/10-split"];
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_60_20_20 = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 96.50 %
split_models_accuracies = [split_models_accuracies, accuracy_60_20_20]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large_70_30, ...
'Plots','training-progress');
accuracy_70_15_15 = model_optimization_explorer (trainingSet_large_70_15_15, validationSet_large_70_15_15, testSet_large_70_15_15, model_options, layers);
The validation accuracy is: 94.00 %
split_models_accuracies = [split_models_accuracies, accuracy_70_15_15]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large_80_10_10, ...
'Plots','training-progress');
accuracy_80_10_10 = model_optimization_explorer (trainingSet_large_80_10_10, validationSet_large_80_10_10, testSet_large_80_10_10, model_options, layers);
The validation accuracy is: 96.00 %
split_models_accuracies = [split_models_accuracies, accuracy_80_10_10]; % Capture accuracy value
disp(split_models)
"60/20/20-split" "70/15/15-split" "80/10/10-split"
disp(split_models_accuracies)
% Convert arrays to a table
dataframe_split_models = table(split_models', split_models_accuracies', 'VariableNames', {'model', 'accuracy'});
disp(dataframe_split_models);
model accuracy
________________ ________
"60/20/20-split" 0.965
"70/15/15-split" 0.94
"80/10/10-split" 0.96
% Variable To Be Explored: optimizer 'sgdm' versus optimizer 'adam'
% Variable Selected/Locked After Testing: 60 / 20 / 20 <-------> training set / validation set / test set split.
% Variable Selected/Locked After Testing: MaxEpochs held at 10
% Variable Selected/Locked After Testing: Use large, UN-augmented data set
% All other variables held constant
% Use established "layers" modified pretrained alexnet network
optimizer_models_accuracies = [];
optimizer_models = ["sdgm", "adam"];
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_60_20_20_sgdm = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 96.00 %
optimizer_models_accuracies = [optimizer_models_accuracies, accuracy_60_20_20_sgdm]; % Capture accuracy value
model_options = trainingOptions('adam', ...
'InitialLearnRate',1e-4, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_60_20_20_adam = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 93.00 %
optimizer_models_accuracies = [optimizer_models_accuracies, accuracy_60_20_20_adam]; % Capture accuracy value
disp(optimizer_models_accuracies)
% Convert arrays to a table
dataframe_optimizer_models = table(optimizer_models', optimizer_models_accuracies', 'VariableNames', {'model', 'accuracy'});
disp(dataframe_optimizer_models);
model accuracy
______ ________
"sdgm" 0.96
"adam" 0.93
% Variable To Be Explored: Try several learning rates (LR) -- 0.00055, 0.0001*, 0.000055, 0.00001
% Original Learning Rate: 0.0001* or 1e-4*
% Variable Selected/Locked After Testing: 60 / 20 / 20 <-------> training set / validation set / test set split.
% Variable Selected/Locked After Testing: MaxEpochs held at 10
% Variable Selected/Locked After Testing: Use large, UN-augmented data set
% Variable Selected/Locked After Testing: Use optimizer 'sgdm'
% All other variables held constant
% Use established "layers" modified pretrained alexnet network
LR_models_accuracies = [];
LR_models = ["LR = 0.00055", "LR = 0.0001", "LR = 0.000055", "LR = 0.00001"];
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.00055, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_LR_0_00055 = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 50.00 %
LR_models_accuracies = [LR_models_accuracies, accuracy_LR_0_00055]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_LR_0_0001 = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 96.50 %
LR_models_accuracies = [LR_models_accuracies, accuracy_LR_0_0001]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.000055, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_LR_0_000055 = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 96.00 %
LR_models_accuracies = [LR_models_accuracies, accuracy_LR_0_000055]; % Capture accuracy value
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.00001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_LR_0_00001 = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 96.50 %
LR_models_accuracies = [LR_models_accuracies, accuracy_LR_0_00001]; % Capture accuracy value
disp(LR_models)
"LR = 0.00055" "LR = 0.0001" "LR = 0.000055" "LR = 0.00001"
disp(LR_models_accuracies)
0.5000 0.9650 0.9600 0.9650
% Convert arrays to a table
dataframe_LR_models = table(LR_models', LR_models_accuracies', 'VariableNames', {'model', 'accuracy'});
disp(dataframe_LR_models);
model accuracy
_______________ ________
"LR = 0.00055" 0.5
"LR = 0.0001" 0.965
"LR = 0.000055" 0.96
"LR = 0.00001" 0.965
% Data augmentation: baseline
imageAugmenter1 = imageDataAugmenter( ...
'RandXReflection',true, ...
'RandXTranslation',pixelRange1, ...
'RandYTranslation',pixelRange1, ...
'RandXScale',scaleRange1, ...
'RandYScale',scaleRange1);
% Data augmentation: increased pixel range
imageAugmenter2 = imageDataAugmenter( ...
'RandXReflection',true, ...
'RandXTranslation',pixelRange2, ...
'RandYTranslation',pixelRange2, ...
'RandXScale',scaleRange2, ...
'RandYScale',scaleRange2);
% Data augmentation: decreased pixel range
imageAugmenter3 = imageDataAugmenter( ...
'RandXReflection',true, ...
'RandXTranslation',pixelRange3, ...
'RandYTranslation',pixelRange3, ...
'RandXScale',scaleRange3, ...
'RandYScale',scaleRange3);
% Data augmentation: increased scale range
scaleRange4 = [0.85 1.15];
imageAugmenter4 = imageDataAugmenter( ...
'RandXReflection',true, ...
'RandXTranslation',pixelRange4, ...
'RandYTranslation',pixelRange4, ...
'RandXScale',scaleRange4, ...
'RandYScale',scaleRange4);
% Data augmentation: decreased scale range
scaleRange5 = [0.95 1.05];
imageAugmenter5 = imageDataAugmenter( ...
'RandXReflection',true, ...
'RandXTranslation',pixelRange5, ...
'RandYTranslation',pixelRange5, ...
'RandXScale',scaleRange5, ...
'RandYScale',scaleRange5);
% Define augmented training and validation data sets
% retrieve the input size of the first layer of the pretrained model
inputSize = model1.Layers(1).InputSize;
% Create an augmented image datastore data set 60/20/20 splits
% Only the training set will undergo the 5 different augmentations defined above
augimdsTrain_large_60_20_20_ag1 = augmentedImageDatastore(inputSize(1:2),trainingSet_large_60_20_20, DataAugmentation = imageAugmenter1);
augimdsTrain_large_60_20_20_ag2 = augmentedImageDatastore(inputSize(1:2),trainingSet_large_60_20_20, DataAugmentation = imageAugmenter2);
augimdsTrain_large_60_20_20_ag3 = augmentedImageDatastore(inputSize(1:2),trainingSet_large_60_20_20, DataAugmentation = imageAugmenter3);
augimdsTrain_large_60_20_20_ag4 = augmentedImageDatastore(inputSize(1:2),trainingSet_large_60_20_20, DataAugmentation = imageAugmenter4);
augimdsTrain_large_60_20_20_ag5 = augmentedImageDatastore(inputSize(1:2),trainingSet_large_60_20_20, DataAugmentation = imageAugmenter5);
validationSet_large_60_20_20;
augmentation_models_accuracies = [];
augmentation_models = ["No Augmentation", "Augmentation_1: baseline", ...
"Augmentation_2: inc pixel range", "Augmentation_3: dec pixel range", ...
"Augmentation_4: inc scale", "Augmentation_5: dec scale"];
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_no_aug = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 97.00 %
augmentation_models_accuracies = [augmentation_models_accuracies, accuracy_no_aug]; % Capture accuracy value
% Data augmentation: baseline
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_ag1 = model_optimization_explorer (augimdsTrain_large_60_20_20_ag1, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 93.50 %
augmentation_models_accuracies = [augmentation_models_accuracies, accuracy_ag1]; % Capture accuracy value
% Data augmentation: increased pixel range
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_ag2 = model_optimization_explorer (augimdsTrain_large_60_20_20_ag2, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 97.00 %
augmentation_models_accuracies = [augmentation_models_accuracies, accuracy_ag2]; % Capture accuracy value
% Data augmentation: decreased pixel range
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_ag3 = model_optimization_explorer (augimdsTrain_large_60_20_20_ag3, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 93.50 %
augmentation_models_accuracies = [augmentation_models_accuracies, accuracy_ag3]; % Capture accuracy value
% Data augmentation: increased scale range
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_ag4 = model_optimization_explorer (augimdsTrain_large_60_20_20_ag4, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 96.00 %
augmentation_models_accuracies = [augmentation_models_accuracies, accuracy_ag4]; % Capture accuracy value
% Data augmentation: decreased scale range
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_ag5 = model_optimization_explorer (augimdsTrain_large_60_20_20_ag5, validationSet_large_60_20_20, testSet_large_60_20_20, model_options, layers);
The validation accuracy is: 96.00 %
augmentation_models_accuracies = [augmentation_models_accuracies, accuracy_ag5]; % Capture accuracy value
disp(augmentation_models)
"No Augmentation" "Augmentation_1: baseline"
"Augmentation_2 inc pixel range" "Augmentation_3: dec pixel range"
"Augmentation_4: inc scale" "Augmentation_5: dec scale"
disp(augmentation_models_accuracies)
0.9700 0.9350 0.9700 0.9350 0.9600 0.9600
% Convert arrays to a table
augmentation_models = ["No Augmentation", "Augmentation_1: baseline", ...
"Augmentation_2: inc pixel range", "Augmentation_3: dec pixel range", ...
"Augmentation_4: inc scale", "Augmentation_5: dec scale"];
dataframe_augmentation_models = table(augmentation_models', augmentation_models_accuracies', ...
'VariableNames', {'model', 'accuracy'});
disp(dataframe_augmentation_models);
model accuracy
_________________________________ ________
"No Augmentation" 0.97
"Augmentation_1: baseline" 0.935
"Augmentation_2: inc pixel range" 0.97
"Augmentation_3: dec pixel range" 0.935
"Augmentation_4: inc scale" 0.96
"Augmentation_5: dec scale" 0.96
% split data into 60 % for training set, 20 % for validation set, 20 % for test set
[trainingSet_large_60_20_20, remainingSet_large_60_20_20] = splitEachLabel(imds_large, 0.6, 'randomized');
[testSet_large_60_20_20, validationSet_large_60_20_20] = splitEachLabel(remainingSet_large_60_20_20, 0.5, 'randomized');
% display number of training set images
countEachLabel(trainingSet_large_60_20_20)
% display number of validation set images
countEachLabel(validationSet_large_60_20_20)
% display number of validation set images
countEachLabel(testSet_large_60_20_20)
% Use pretrained googlenet "goog_net"
% Modify the last 3 layers of goog_net for specific classification task
%--Show current last three layers of goog_net
current_layers = goog_net.Layers(end-2:end)
current_layers =
3×1 Layer array with layers:
1 'loss3-classifier' Fully Connected 1000 fully connected layer
2 'prob' Softmax softmax
3 'output' Classification Output crossentropyex with 'tench' and 999 other classes
%--Define needed last three layers of goog_net
fullyConnectedLayer(number_of_classes,'Name','fc','WeightLearnRateFactor',10,'BiasLearnRateFactor',10)
softmaxLayer('Name','softmax')
classificationLayer('Name','output')];
%--Show needed last three layers of goog_net
new_layers
new_layers =
3×1 Layer array with layers:
1 'fc' Fully Connected 2 fully connected layer
2 'softmax' Softmax softmax
3 'output' Classification Output crossentropyex
%--Replace the last layers in goog_net
goog_net = layerGraph(goog_net);
goog_net = replaceLayer(goog_net,'loss3-classifier',new_layers(1));
goog_net = replaceLayer(goog_net,'prob',new_layers(2));
goog_net = replaceLayer(goog_net,'output',new_layers(3));
%--Show newly replaced last three layers of goog_net
current_layers = goog_net.Layers(end-2:end)
current_layers =
3×1 Layer array with layers:
1 'fc' Fully Connected 2 fully connected layer
2 'softmax' Softmax softmax
3 'output' Classification Output crossentropyex
% Use pretrained googlenet "goog_net"
model_options = trainingOptions('sgdm', ...
'InitialLearnRate',0.0001, ...
'ValidationData',validationSet_large_60_20_20, ...
'Plots','training-progress');
accuracy_googlenet = model_optimization_explorer (trainingSet_large_60_20_20, validationSet_large_60_20_20, ...
testSet_large_60_20_20, model_options, goog_net);
The validation accuracy is: 99.00 %
pretrained_models_accuracies = [pretrained_models_accuracies, accuracy_googlenet]; % Capture accuracy value
function [prediction_accuracy] = model_optimization_explorer (training_data, validation_data, test_data, model_options, network_layers)
predictive_model = trainNetwork(training_data, network_layers, model_options);
% The YPredAug variable holds the predicted labels for the images in the augmented validationSet.
% The probsAug variable contains the scores associated with each class for each image in the augmented validationSet.
% It is a matrix where each row corresponds to an image in the augmented validationSet, and each column corresponds to a class.
% The values in scores represent the confidence or probability of each class for each image.
[predicted_labels,label_probabilities] = classify(predictive_model,test_data);
true_labels = test_data.Labels; % determine actual validation image set labels
% Compute fraction of correctly predicted labels
% by comparing predicted versus actual augmented validation image set labels
prediction_accuracy = mean(predicted_labels == true_labels);
fprintf("The validation accuracy is: %.2f %%\n", prediction_accuracy * 100);
class_names = unique([true_labels; predicted_labels]);
% Compute the confusion matrix
cf_matrix = confusionmat(true_labels, predicted_labels, 'Order', class_names);
% Visualize the confusion matrix as a heatmap
heatmap(class_names, class_names, cf_matrix);
title('Confusion Matrix');
xlabel('Predicted Labels');
% Inspect incorrect classifications
% Find indices where the true labels and predicted labels do not match
incorrect_indices = find(true_labels ~= predicted_labels);
if isempty(incorrect_indices)
disp("No images misclassified")
% Display sample of the incorrectly classified images
image_sample = min(20, numel(incorrect_indices)); % Show at most 20 images
idx = incorrect_indices(i);
% Extract the file path from the cell array
file_path_cell = validation_data.Files(idx);
% Convert the cell array into a string
file_path_string = char(file_path_cell);
% Show image with true and predicted labels
img = imread(file_path_string);
img_resized = imresize(img, [400, 600]);
title(sprintf('True: %s, \n Predicted: %s', true_labels(idx), predicted_labels(idx) ));